


BOT_NAME = 'cnblog'

SPIDER_MODULES = ['cnblog.spiders']
NEWSPIDER_MODULE = 'cnblog.spiders'

#一定要把这个参数的值更改为False
ROBOTSXT_OBEY = False

DEFAULT_REQUEST_HEADERS = {
 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
   'Accept-Language': 'en',
    #user-agent新添加
    'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"

}

#新修改
ITEM_PIPELINES = {
    #实现保存到txt文件
    'cnblog.pipelines.FilePipeline: 300',
}